
# initialize
library(tm)
source("./config")

#### load Reuters-21578 ###
reuters <- Corpus(DirSource(corpusDir), readerControl = list(reader = readReut21578XML))

### prepare loaded data ###

# convert to Plain Text Documents
reuters <- tm_map(reuters, as.PlainTextDocument)

# convert to Lower Case
reuters <- tm_map(reuters, tolower)

# remove Stopwords
reuters <- tm_map(reuters, removeWords, stopwords("english"))

# remove Punctuations
reuters <- tm_map(reuters, removePunctuation)

# stemming
reuters <- tm_map(reuters, stemDocument)

# remove Numbers
reuters <- tm_map(reuters, removeNumbers)

# eliminating Extra White Spaces
reuters <- tm_map(reuters, stripWhitespace)

# remove empty elements
reuters <- reuters[unlist(lapply(reuters, length) != 0)]

# term matrix and convert it to actual matrix
dtm <- DocumentTermMatrix(reuters)

dtm <- removeSparseTerms(dtm, sparseTermRm)

if (weightTermFreq) {
  dtm <- weightTfIdf(dtm)
}

fdtm <- data.frame(data.matrix(dtm))

# construct new data frame
r.train <- fdtm[0,]
r.train["Topic"] <- character(0)

r.test <- fdtm[0,]
r.test["Topic"] <- character(0)

j.ts <- 1
j.tr <- 1
ncl <- dim(fdtm)[2]

for (i in 1:length(reuters)) {
    print(i)

    # assign "OTHER" to docs without any topic
    if (length(attr(reuters[[i]],"LocalMetaData")$Topics) == 0) {
        attr(reuters[[i]],"LocalMetaData")$Topics <- "OTHER"
    }

    # split train and test sets
    if (attr(reuters[[i]],"LocalMetaData")$LEWISSPLIT == "TRAIN") {
        # trim redundant classes (each doc should have one class)
        r.train[j.tr,1:ncl] <- as.numeric(fdtm[i,])
        r.train[j.tr,"Topic"] <- attr(reuters[[i]],"LocalMetaData")$Topics[1]
        j.tr <- j.tr+1

    }
    else {
        # trim redundant classes (each doc should have one class)
        r.test[j.ts,1:ncl] <- as.numeric(fdtm[i,])
        r.test[j.ts,"Topic"] <- attr(reuters[[i]],"LocalMetaData")$Topics[1]
        j.ts <- j.ts+1
    }
}

# change topics to factor
r.train[,"Topic"] <- as.factor(r.train[,"Topic"])
r.test[,"Topic"] <- as.factor(r.test[,"Topic"])

# number of attributes
n = dim(r.train)[2]

